{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gzip\n",
    "import subprocess\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        yield eval(l)\n",
    "\n",
    "def get_df(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET = 'Grocery_and_Gourmet_Food'\n",
    "RAW_PATH = os.path.join('./', DATASET)\n",
    "DATA_FILE = 'reviews_{}_5.json.gz'.format(DATASET)\n",
    "META_FILE = 'meta_{}.json.gz'.format(DATASET)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Load interaction data and item metadata\n",
    "2. Filter out unuseful items\n",
    "3. Calculate basic statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading interaction data into ./Grocery_and_Gourmet_Food\n",
      "Downloading item metadata into ./Grocery_and_Gourmet_Food\n"
     ]
    }
   ],
   "source": [
    "# download data if not exists\n",
    "\n",
    "if not os.path.exists(RAW_PATH):\n",
    "    subprocess.call('mkdir ' + RAW_PATH, shell=True)\n",
    "if not os.path.exists(os.path.join(RAW_PATH, DATA_FILE)):\n",
    "    print('Downloading interaction data into ' + RAW_PATH)\n",
    "    subprocess.call(\n",
    "        'cd {} && curl -O http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_{}_5.json.gz'\n",
    "        .format(RAW_PATH, DATASET), shell=True)\n",
    "if not os.path.exists(os.path.join(RAW_PATH, META_FILE)):\n",
    "    print('Downloading item metadata into ' + RAW_PATH)\n",
    "    subprocess.call(\n",
    "        'cd {} && curl -O http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/meta_{}.json.gz'\n",
    "        .format(RAW_PATH, DATASET), shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>helpful</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>overall</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>reviewTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1VEELTKS8NLZB</td>\n",
       "      <td>616719923X</td>\n",
       "      <td>Amazon Customer</td>\n",
       "      <td>[0, 0]</td>\n",
       "      <td>Just another flavor of Kit Kat but the taste i...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good Taste</td>\n",
       "      <td>1370044800</td>\n",
       "      <td>06 1, 2013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A14R9XMZVJ6INB</td>\n",
       "      <td>616719923X</td>\n",
       "      <td>amf0001</td>\n",
       "      <td>[0, 1]</td>\n",
       "      <td>I bought this on impulse and it comes from Jap...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5 stars,  sadly not as wonderful as I had hoped</td>\n",
       "      <td>1400457600</td>\n",
       "      <td>05 19, 2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A27IQHDZFQFNGG</td>\n",
       "      <td>616719923X</td>\n",
       "      <td>Caitlin</td>\n",
       "      <td>[3, 4]</td>\n",
       "      <td>Really good. Great gift for any fan of green t...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Yum!</td>\n",
       "      <td>1381190400</td>\n",
       "      <td>10 8, 2013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A31QY5TASILE89</td>\n",
       "      <td>616719923X</td>\n",
       "      <td>DebraDownSth</td>\n",
       "      <td>[0, 0]</td>\n",
       "      <td>I had never had it before, was curious to see ...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Unexpected flavor meld</td>\n",
       "      <td>1369008000</td>\n",
       "      <td>05 20, 2013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2LWK003FFMCI5</td>\n",
       "      <td>616719923X</td>\n",
       "      <td>Diana X.</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>I've been looking forward to trying these afte...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Not a very strong tea flavor, but still yummy ...</td>\n",
       "      <td>1369526400</td>\n",
       "      <td>05 26, 2013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       reviewerID        asin     reviewerName helpful  \\\n",
       "0  A1VEELTKS8NLZB  616719923X  Amazon Customer  [0, 0]   \n",
       "1  A14R9XMZVJ6INB  616719923X          amf0001  [0, 1]   \n",
       "2  A27IQHDZFQFNGG  616719923X          Caitlin  [3, 4]   \n",
       "3  A31QY5TASILE89  616719923X     DebraDownSth  [0, 0]   \n",
       "4  A2LWK003FFMCI5  616719923X         Diana X.  [1, 2]   \n",
       "\n",
       "                                          reviewText  overall  \\\n",
       "0  Just another flavor of Kit Kat but the taste i...      4.0   \n",
       "1  I bought this on impulse and it comes from Jap...      3.0   \n",
       "2  Really good. Great gift for any fan of green t...      4.0   \n",
       "3  I had never had it before, was curious to see ...      5.0   \n",
       "4  I've been looking forward to trying these afte...      4.0   \n",
       "\n",
       "                                             summary  unixReviewTime  \\\n",
       "0                                         Good Taste      1370044800   \n",
       "1  3.5 stars,  sadly not as wonderful as I had hoped      1400457600   \n",
       "2                                               Yum!      1381190400   \n",
       "3                             Unexpected flavor meld      1369008000   \n",
       "4  Not a very strong tea flavor, but still yummy ...      1369526400   \n",
       "\n",
       "    reviewTime  \n",
       "0   06 1, 2013  \n",
       "1  05 19, 2014  \n",
       "2   10 8, 2013  \n",
       "3  05 20, 2013  \n",
       "4  05 26, 2013  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df = get_df(os.path.join(RAW_PATH, DATA_FILE))\n",
    "data_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>imUrl</th>\n",
       "      <th>related</th>\n",
       "      <th>salesRank</th>\n",
       "      <th>categories</th>\n",
       "      <th>price</th>\n",
       "      <th>brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0657745316</td>\n",
       "      <td>This is real vanilla extract made with only 3 ...</td>\n",
       "      <td>100 Percent All Natural Vanilla Extract</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41gFi5h0...</td>\n",
       "      <td>{'also_viewed': ['B001GE8N4Y']}</td>\n",
       "      <td>{'Grocery &amp; Gourmet Food': 374004}</td>\n",
       "      <td>[[Grocery &amp; Gourmet Food]]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0700026444</td>\n",
       "      <td>Silverpot Tea, Pure Darjeeling, is an exquisit...</td>\n",
       "      <td>Pure Darjeeling Tea: Loose Leaf</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51hs8sox...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'Grocery &amp; Gourmet Food': 620307}</td>\n",
       "      <td>[[Grocery &amp; Gourmet Food]]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1403796890</td>\n",
       "      <td>Must have for any WWE Fan\\n \\n \\n \\nFeaturing ...</td>\n",
       "      <td>WWE Kids Todler Velvet Slippers featuring John...</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/518SEST5...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Grocery &amp; Gourmet Food]]</td>\n",
       "      <td>3.99</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>141278509X</td>\n",
       "      <td>Infused with Vitamins and Electrolytes Good So...</td>\n",
       "      <td>Archer Farms Strawberry Dragonfruit Drink Mix ...</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51CFQIis...</td>\n",
       "      <td>{'also_viewed': ['B0051IETTY']}</td>\n",
       "      <td>{'Grocery &amp; Gourmet Food': 620322}</td>\n",
       "      <td>[[Grocery &amp; Gourmet Food]]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1453060375</td>\n",
       "      <td>MiO Energy is your portable energy source givi...</td>\n",
       "      <td>Mio Energy Liquid Water Enhancer Black Cherry ...</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51EUsMcn...</td>\n",
       "      <td>{'also_viewed': ['B006MSEOJ2', 'B005VOOQLO', '...</td>\n",
       "      <td>{'Grocery &amp; Gourmet Food': 268754}</td>\n",
       "      <td>[[Grocery &amp; Gourmet Food]]</td>\n",
       "      <td>11.99</td>\n",
       "      <td>Mio</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         asin                                        description  \\\n",
       "0  0657745316  This is real vanilla extract made with only 3 ...   \n",
       "1  0700026444  Silverpot Tea, Pure Darjeeling, is an exquisit...   \n",
       "2  1403796890  Must have for any WWE Fan\\n \\n \\n \\nFeaturing ...   \n",
       "3  141278509X  Infused with Vitamins and Electrolytes Good So...   \n",
       "4  1453060375  MiO Energy is your portable energy source givi...   \n",
       "\n",
       "                                               title  \\\n",
       "0            100 Percent All Natural Vanilla Extract   \n",
       "1                    Pure Darjeeling Tea: Loose Leaf   \n",
       "2  WWE Kids Todler Velvet Slippers featuring John...   \n",
       "3  Archer Farms Strawberry Dragonfruit Drink Mix ...   \n",
       "4  Mio Energy Liquid Water Enhancer Black Cherry ...   \n",
       "\n",
       "                                               imUrl  \\\n",
       "0  http://ecx.images-amazon.com/images/I/41gFi5h0...   \n",
       "1  http://ecx.images-amazon.com/images/I/51hs8sox...   \n",
       "2  http://ecx.images-amazon.com/images/I/518SEST5...   \n",
       "3  http://ecx.images-amazon.com/images/I/51CFQIis...   \n",
       "4  http://ecx.images-amazon.com/images/I/51EUsMcn...   \n",
       "\n",
       "                                             related  \\\n",
       "0                    {'also_viewed': ['B001GE8N4Y']}   \n",
       "1                                                NaN   \n",
       "2                                                NaN   \n",
       "3                    {'also_viewed': ['B0051IETTY']}   \n",
       "4  {'also_viewed': ['B006MSEOJ2', 'B005VOOQLO', '...   \n",
       "\n",
       "                            salesRank                  categories  price brand  \n",
       "0  {'Grocery & Gourmet Food': 374004}  [[Grocery & Gourmet Food]]    NaN   NaN  \n",
       "1  {'Grocery & Gourmet Food': 620307}  [[Grocery & Gourmet Food]]    NaN   NaN  \n",
       "2                                 NaN  [[Grocery & Gourmet Food]]   3.99   NaN  \n",
       "3  {'Grocery & Gourmet Food': 620322}  [[Grocery & Gourmet Food]]    NaN   NaN  \n",
       "4  {'Grocery & Gourmet Food': 268754}  [[Grocery & Gourmet Food]]  11.99   Mio  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_df = get_df(os.path.join(RAW_PATH, META_FILE))\n",
    "meta_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter items\n",
    "\n",
    "useful_meta_df = meta_df[meta_df['asin'].isin(data_df['asin'])]\n",
    "items_with_info = useful_meta_df['related'].apply(lambda x: x is not np.nan)\n",
    "useful_meta_df = useful_meta_df[items_with_info].reset_index(drop=True)\n",
    "\n",
    "all_items = set(useful_meta_df['asin'].values.tolist())\n",
    "def related_filter(related_dict):\n",
    "    out_dict = dict()\n",
    "    for r in related_dict:\n",
    "        out_dict[r] = list(all_items & set(related_dict[r]))\n",
    "    return out_dict\n",
    "\n",
    "useful_meta_df['related'] = useful_meta_df['related'].apply(related_filter)\n",
    "data_df = data_df[data_df['asin'].isin(all_items)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_users = data_df['reviewerID'].value_counts().size\n",
    "n_items = data_df['asin'].value_counts().size\n",
    "n_clicks = len(data_df)\n",
    "min_time = data_df['unixReviewTime'].min()\n",
    "max_time = data_df['unixReviewTime'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Users: 14680\n",
      "# Items: 8456\n",
      "# Interactions: 145832\n",
      "Time Span: 2000-08-09/2014-07-23\n"
     ]
    }
   ],
   "source": [
    "time_format = '%Y-%m-%d'\n",
    "\n",
    "print('# Users:', n_users)\n",
    "print('# Items:', n_items)\n",
    "print('# Interactions:', n_clicks)\n",
    "print('Time Span: {}/{}'.format(\n",
    "    datetime.utcfromtimestamp(min_time).strftime(time_format),\n",
    "    datetime.utcfromtimestamp(max_time).strftime(time_format))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interaction data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(2019)\n",
    "NEG_ITEMS = 99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>A1KXONFPU2XQ5K</td>\n",
       "      <td>B00004S1C5</td>\n",
       "      <td>965779200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>A23GFTVIETX7DS</td>\n",
       "      <td>B0000CH39R</td>\n",
       "      <td>1068249600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>A281NPSIMI1C2R</td>\n",
       "      <td>B0000DBN1H</td>\n",
       "      <td>1073433600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>A3M174IC0VXOS2</td>\n",
       "      <td>B0000537AF</td>\n",
       "      <td>1075593600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1413</th>\n",
       "      <td>A218J1WI08045B</td>\n",
       "      <td>B0001EQN88</td>\n",
       "      <td>1082073600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             user_id     item_id        time\n",
       "35    A1KXONFPU2XQ5K  B00004S1C5   965779200\n",
       "136   A23GFTVIETX7DS  B0000CH39R  1068249600\n",
       "171   A281NPSIMI1C2R  B0000DBN1H  1073433600\n",
       "55    A3M174IC0VXOS2  B0000537AF  1075593600\n",
       "1413  A218J1WI08045B  B0001EQN88  1082073600"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df = data_df.rename(columns={'asin': 'item_id', 'reviewerID': 'user_id', 'unixReviewTime': 'time'})\n",
    "out_df = out_df[['user_id', 'item_id', 'time']]\n",
    "out_df = out_df.drop_duplicates(['user_id', 'item_id', 'time'])\n",
    "out_df.sort_values(by=['time', 'user_id', 'item_id'], inplace=True)\n",
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2177</td>\n",
       "      <td>3</td>\n",
       "      <td>965779200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4161</td>\n",
       "      <td>18</td>\n",
       "      <td>1068249600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4698</td>\n",
       "      <td>23</td>\n",
       "      <td>1073433600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10145</td>\n",
       "      <td>6</td>\n",
       "      <td>1075593600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3915</td>\n",
       "      <td>125</td>\n",
       "      <td>1082073600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id        time\n",
       "0     2177        3   965779200\n",
       "1     4161       18  1068249600\n",
       "2     4698       23  1073433600\n",
       "3    10145        6  1075593600\n",
       "4     3915      125  1082073600"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reindex (start from 1)\n",
    "\n",
    "uids = sorted(out_df['user_id'].unique())\n",
    "user2id = dict(zip(uids, range(1, len(uids) + 1)))\n",
    "iids = sorted(out_df['item_id'].unique())\n",
    "item2id = dict(zip(iids, range(1, len(iids) + 1)))\n",
    "\n",
    "out_df['user_id'] = out_df['user_id'].apply(lambda x: user2id[x])\n",
    "out_df['item_id'] = out_df['item_id'].apply(lambda x: item2id[x])\n",
    "out_df = out_df.reset_index(drop=True)\n",
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# leave one out spliting\n",
    "\n",
    "clicked_item_set = dict()\n",
    "for user_id, seq_df in out_df.groupby('user_id'):\n",
    "    clicked_item_set[user_id] = set(seq_df['item_id'].values.tolist())\n",
    "    \n",
    "def generate_dev_test(data_df):\n",
    "    result_dfs = []\n",
    "    for idx in range(2):\n",
    "        result_df = data_df.groupby('user_id').tail(1).copy()\n",
    "        data_df = data_df.drop(result_df.index)\n",
    "        neg_items = np.random.randint(1, len(iids) + 1, (len(result_df), NEG_ITEMS))\n",
    "        for i, uid in enumerate(result_df['user_id'].values):\n",
    "            user_clicked = clicked_item_set[uid]\n",
    "            for j in range(len(neg_items[i])):\n",
    "                while neg_items[i][j] in user_clicked:\n",
    "                    neg_items[i][j] = np.random.randint(1, len(iids) + 1)\n",
    "        result_df['neg_items'] = neg_items.tolist()\n",
    "        result_dfs.append(result_df)\n",
    "    return result_dfs, data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(116489, 14667, 14676)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "leave_df = out_df.groupby('user_id').head(1)\n",
    "data_df = out_df.drop(leave_df.index)\n",
    "\n",
    "[test_df, dev_df], data_df = generate_dev_test(data_df)\n",
    "train_df = pd.concat([leave_df, data_df]).sort_index()\n",
    "\n",
    "len(train_df), len(dev_df), len(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2177</td>\n",
       "      <td>3</td>\n",
       "      <td>965779200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4161</td>\n",
       "      <td>18</td>\n",
       "      <td>1068249600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4698</td>\n",
       "      <td>23</td>\n",
       "      <td>1073433600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10145</td>\n",
       "      <td>6</td>\n",
       "      <td>1075593600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3915</td>\n",
       "      <td>125</td>\n",
       "      <td>1082073600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id        time\n",
       "0     2177        3   965779200\n",
       "1     4161       18  1068249600\n",
       "2     4698       23  1073433600\n",
       "3    10145        6  1075593600\n",
       "4     3915      125  1082073600"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>time</th>\n",
       "      <th>neg_items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>6185</td>\n",
       "      <td>747</td>\n",
       "      <td>1149206400</td>\n",
       "      <td>[7241, 4287, 2073, 7440, 2445, 107, 1041, 1224...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>1282</td>\n",
       "      <td>815</td>\n",
       "      <td>1154044800</td>\n",
       "      <td>[6393, 3250, 5724, 6972, 2478, 3197, 4367, 663...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>3299</td>\n",
       "      <td>1070</td>\n",
       "      <td>1154044800</td>\n",
       "      <td>[3948, 3361, 4492, 6754, 2335, 4144, 3166, 110...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>12800</td>\n",
       "      <td>317</td>\n",
       "      <td>1157328000</td>\n",
       "      <td>[697, 722, 689, 3040, 3408, 2455, 2932, 3855, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>472</th>\n",
       "      <td>10678</td>\n",
       "      <td>504</td>\n",
       "      <td>1163980800</td>\n",
       "      <td>[4644, 4463, 5429, 4961, 7100, 179, 1683, 1201...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  item_id        time  \\\n",
       "102     6185      747  1149206400   \n",
       "190     1282      815  1154044800   \n",
       "192     3299     1070  1154044800   \n",
       "296    12800      317  1157328000   \n",
       "472    10678      504  1163980800   \n",
       "\n",
       "                                             neg_items  \n",
       "102  [7241, 4287, 2073, 7440, 2445, 107, 1041, 1224...  \n",
       "190  [6393, 3250, 5724, 6972, 2478, 3197, 4367, 663...  \n",
       "192  [3948, 3361, 4492, 6754, 2335, 4144, 3166, 110...  \n",
       "296  [697, 722, 689, 3040, 3408, 2455, 2932, 3855, ...  \n",
       "472  [4644, 4463, 5429, 4961, 7100, 179, 1683, 1201...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save results\n",
    "\n",
    "train_df.to_csv(os.path.join(RAW_PATH, 'train.csv'), sep='\\t', index=False)\n",
    "dev_df.to_csv(os.path.join(RAW_PATH, 'dev.csv'), sep='\\t', index=False)\n",
    "test_df.to_csv(os.path.join(RAW_PATH, 'test.csv'), sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Item Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l2 category\n",
    "\n",
    "l2_cate_lst = list()\n",
    "for cate_lst in useful_meta_df['categories']:\n",
    "    l2_cate_lst.append(cate_lst[0][2] if len(cate_lst[0]) > 2 else np.nan)\n",
    "useful_meta_df['l2_category'] = l2_cate_lst  \n",
    "l2_cates = sorted(useful_meta_df['l2_category'].dropna().unique())\n",
    "l2_dict = dict(zip(l2_cates, range(1, len(l2_cates) + 1)))\n",
    "useful_meta_df['l2_category'] = useful_meta_df['l2_category'].apply(lambda x: l2_dict[x] if x == x else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>i_category</th>\n",
       "      <th>r_complement</th>\n",
       "      <th>r_substitute</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[1502, 6077, 5940, 285, 5122, 6211, 176, 5866,...</td>\n",
       "      <td>[5866, 6077, 4006, 2212, 193, 6211, 5985]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[2280, 1698, 4033, 825, 5045, 5681, 2488, 212,...</td>\n",
       "      <td>[823, 4751, 1599, 824, 994, 91, 2120, 826, 827...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>[2130, 2535, 4697, 7348, 160, 2792, 140, 2065]</td>\n",
       "      <td>[4383, 16, 5630, 108]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>[7666, 8205, 1688, 6620, 5539, 7295, 7047, 498...</td>\n",
       "      <td>[4669, 7047, 4231]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[471, 353, 366, 368, 484, 5398, 681, 367, 479,...</td>\n",
       "      <td>[351, 347, 350, 358, 353, 352, 366, 359, 356, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id  i_category                                       r_complement  \\\n",
       "0        1           0  [1502, 6077, 5940, 285, 5122, 6211, 176, 5866,...   \n",
       "1        2           0  [2280, 1698, 4033, 825, 5045, 5681, 2488, 212,...   \n",
       "2        3          20     [2130, 2535, 4697, 7348, 160, 2792, 140, 2065]   \n",
       "3        4           0  [7666, 8205, 1688, 6620, 5539, 7295, 7047, 498...   \n",
       "4        5           0  [471, 353, 366, 368, 484, 5398, 681, 367, 479,...   \n",
       "\n",
       "                                        r_substitute  \n",
       "0          [5866, 6077, 4006, 2212, 193, 6211, 5985]  \n",
       "1  [823, 4751, 1599, 824, 994, 91, 2120, 826, 827...  \n",
       "2                              [4383, 16, 5630, 108]  \n",
       "3                                 [4669, 7047, 4231]  \n",
       "4  [351, 347, 350, 358, 353, 352, 366, 359, 356, ...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_meta_data = dict()\n",
    "for idx in range(len(useful_meta_df)):\n",
    "    info = useful_meta_df.iloc[idx]['related']\n",
    "    item_meta_data[idx] = {\n",
    "        'item_id': item2id[useful_meta_df.iloc[idx]['asin']],\n",
    "        'i_category': useful_meta_df.iloc[idx]['l2_category'],\n",
    "        'r_complement': list(map(lambda x: item2id[x], info['also_bought'])) if 'also_bought' in info else [],\n",
    "        'r_substitute': list(map(lambda x: item2id[x], info['also_viewed'])) if 'also_viewed' in info else [],\n",
    "    }\n",
    "\n",
    "item_meta_df = pd.DataFrame.from_dict(item_meta_data, orient='index')\n",
    "item_meta_df = item_meta_df[['item_id', 'i_category', 'r_complement', 'r_substitute']]\n",
    "item_meta_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save results\n",
    "\n",
    "item_meta_df.to_csv(os.path.join(RAW_PATH, 'item_meta.csv'), sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
